package json

import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.rdd.RDD

import scala.util.parsing.json.JSON
import org.junit.{After, Before, Test}

/**
 * 除了窄依赖的依赖都是宽依赖
 */
class Driver {

	val conf: SparkConf = new SparkConf().setAppName("Serializ").setMaster("local[3]")
	var sc: SparkContext = null
	var outpath: String = "out"
	import util.MyPredef._

	@Before
	def init() {
		sc = new SparkContext(conf)
		outpath.delete()
	}

	@After
	def after() {
		sc.stop()
	}

	@Test
	def main(): Unit = {
		val json = sc.textFile("in/user.json")
		val user: RDD[Option[Any]] = json.map(JSON.parseFull)
		user.foreach(println)


	}
}
